package org.axeidls.component;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.locks.ReentrantLock;
import java.util.regex.Pattern;

import org.axe.annotation.ioc.Autowired;
import org.axe.annotation.ioc.Component;
import org.axe.annotation.ioc.Value;
import org.axe.exception.LogableException;
import org.axe.extra.cache.AxeCache;
import org.axe.extra.cache.AxeCacheTimeoutListener;
import org.axe.interface_.mvc.Onload;
import org.axe.util.CollectionUtil;
import org.axe.util.FileUtil;
import org.axe.util.JsonUtil;
import org.axe.util.LogUtil;
import org.axe.util.StringUtil;
import org.axeidls.constant.CleanDataTimeMode;
import org.axeidls.util.jeval.Expression;
import org.axeidls.util.jeval.JevalUtil;
import org.axeidls.util.jeval.Variable;
import org.axeidls.vo.CleanDataResult;
import org.axeidls.vo.DataBlockInfo;
import org.axeidls.vo.DataBlockInfoPlus;
import org.axeidls.vo.DataRecord;
import org.axeidls.vo.DataRecordInfo;
import org.axeidls.vo.DataRecordPackage;
import org.axeidls.vo.DataRecordPlus;
import org.axeidls.vo.DeviceDataListByTimestamp2;
import org.axeidls.vo.Page;
import org.axeidls.vo.PageConfig;
import org.axeidls.vo.StorageStatus;
import org.axeidls.vo.StorageStatusPlus;

@Component
public class DataFileComponent implements Onload {

	private ReentrantLock ioCacheLock = new ReentrantLock();
	private boolean released = false;
	@Autowired
	private AxeCache cache;//如果是框架启动，有缓存就用缓存，如果不是，说明是智能终端使用，就不用缓存
	private final int cacheKeepTime=10;//缓存保存时间

	//这两个常量禁止修改，关系到数据结构和插入数据匹配，数据导出
	private final int indexFileDataUnitLen = 16;
	private final String colSplitFlag = ",";//列分隔符
	private byte[] returnAndEnterBytes = null;
	{
		try {
			returnAndEnterBytes = "\r\n".getBytes("utf-8");
		} catch (Exception e) {
			LogUtil.error(e);
			System.exit(0);
		}
	}

	@Value("${data.file-path}")
	private String dataFilePath;

	@Value("${data.file-size-max}")
	private long dataFileSizeMaxBytes = 0;//单位B

	
	public Set<String> getDevicePathAll() {
		Set<String> set = new HashSet<>();
		buildDevicePathAll("", set);
		return set;
	}
	private void buildDevicePathAll(String devicePath,Set<String> set) {
		List<Map<String, Object>> listDataFilePath = listDataFilePath(devicePath);
		boolean devicePathOk = false;
		for(Map<String,Object> row:listDataFilePath) {
			String fileName = row.get("fileName").toString();
			if(fileName.equals("devicePath.flag")) {
				devicePathOk = true;
				break;
			}
		}
		if(devicePathOk) {
			set.add(devicePath);
			return;
		}
		for(Map<String,Object> row:listDataFilePath) {
			String fileName = row.get("fileName").toString();
			if(!row.get("isDirectory").toString().equalsIgnoreCase("true")) {
				continue;
			}
			String nextDataPath = null;
			if(StringUtil.isEmpty(devicePath)) {
				nextDataPath = fileName.toString();
			}else {
				nextDataPath = devicePath+"/"+fileName.toString();
			}
			buildDevicePathAll(nextDataPath, set);
		}
	}
	
	public long getFreeSpace() {
		return new File(dataFilePath).getFreeSpace();
	}
	
	public void release() {
		ioCacheLock.lock();
		released = true;
		try {
			List<String> listKeyLike = cache.listKeyLike("_FILE_RW:");
			for(String key:listKeyLike) {
				OutputStream io = cache.getValueByKey(key);
				if(io != null) {
					try {
						io.close();
					} catch (Exception e) {}
				}
				cache.delKeyValue(key);
			}
		} catch (Exception e) {
			LogUtil.error(e);
		} finally {
			ioCacheLock.unlock();
		}
		
		if(!cache.getAxeIocMode()) {
			cache.stopAsync();
		}
	}
	
	public CleanDataResult cleanDataAllDevicePath(CleanDataTimeMode cleanDataTimeMode,int keepDataNumber) {
		CleanDataResult result = new CleanDataResult();
		if(keepDataNumber < 0) {
			return result;
		}
		HashSet<String> cleanDataDevicePathSet = new HashSet<>();
		File dataFileFolder = new File(dataFilePath);
		int devicePathIndexStart = dataFileFolder.getAbsolutePath().length();
		File[] listFiles = dataFileFolder.listFiles();
		for(File file:listFiles) {
			cleanDataByDevicePath(file, devicePathIndexStart, cleanDataDevicePathSet, cleanDataTimeMode, keepDataNumber,result);
		}
		return result;
	}

	public CleanDataResult cleanDataByDevicePath(Set<String> cleanDataDevicePathSet,CleanDataTimeMode cleanDataTimeMode,int keepDataNumber) {
		CleanDataResult result = new CleanDataResult();
		if(CollectionUtil.isEmpty(cleanDataDevicePathSet)) {
			return result;
		}
		if(keepDataNumber < 0) {
			return result;
		}
		File dataFileFolder = new File(dataFilePath);
		int devicePathIndexStart = dataFileFolder.getAbsolutePath().length();
		File[] listFiles = dataFileFolder.listFiles();
		for(File file:listFiles) {
			cleanDataByDevicePath(file, devicePathIndexStart, cleanDataDevicePathSet, cleanDataTimeMode, keepDataNumber,result);
		}
		return result;
	}
	
	private void cleanDataByDevicePath(File folder,int devicePathIndexStart,Set<String> cleanDataDevicePathSet,CleanDataTimeMode cleanDataTimeMode,int keepDataNumber,CleanDataResult result) {
		File devicePathFlagFile = new File(folder,"devicePath.flag");
		if(devicePathFlagFile.exists()) {//说明确实是设备数据层
			String devicePath = folder.getAbsolutePath().substring(devicePathIndexStart);
			do {
				//到达数据层，检查是不是要清理的设备，如果不是，则跳过
				if(cleanDataDevicePathSet.size() > 0 && !cleanDataDevicePathSet.contains(devicePath)) {
					break;
				}
				
				//如果是，则进行统计清理
				List<String> listDataYearFolder = listDataYearFolder(devicePath);
				if(cleanDataTimeMode.equals(CleanDataTimeMode.YEAR)) {
					int deleteNumber = listDataYearFolder.size()-keepDataNumber;
					for(int i=0;i<deleteNumber;i++) {
						String deleteDataYear = listDataYearFolder.get(i);
						File yearFolder = new File(folder,deleteDataYear);
						deleteFolder(yearFolder.getAbsolutePath(),result);
					}
					break;//年处理结束
				}
				
				if(cleanDataTimeMode.equals(CleanDataTimeMode.MONTH)) {
					List<String[]> listDataYearMonth = new ArrayList<>();
					for(String dataYear:listDataYearFolder) {
						List<String> listDataMonthFolder = listDataMonthFolder(devicePath, dataYear);
						for(String dataMonth:listDataMonthFolder) {
							listDataYearMonth.add(new String[] {dataYear,dataMonth});
						}
					}
					
					int deleteNumber = listDataYearMonth.size()-keepDataNumber;
					for(int i=0;i<deleteNumber;i++) {
						String[] deleteDataYearMonth = listDataYearMonth.get(i);
						File yearFolder = new File(folder,deleteDataYearMonth[0]);
						File monthFolder = new File(yearFolder,deleteDataYearMonth[1]);
						deleteFolder(monthFolder.getAbsolutePath(),result);
					}
					break;//月处理结束
				}
				
				
				if(cleanDataTimeMode.equals(CleanDataTimeMode.DAY)) {
					List<String[]> listDataYearMonthDay = new ArrayList<>();
					for(String dataYear:listDataYearFolder) {
						List<String> listDataMonthFolder = listDataMonthFolder(devicePath, dataYear);
						for(String dataMonth:listDataMonthFolder) {
							List<String> listDataDayFolder = listDataDayFolder(devicePath, dataYear, dataMonth);
							for(String dataDay:listDataDayFolder) {
								listDataYearMonthDay.add(new String[] {dataYear,dataMonth,dataDay});
							}
						}
					}
					
					int deleteNumber = listDataYearMonthDay.size()-keepDataNumber;
					for(int i=0;i<deleteNumber;i++) {
						String[] deleteDataYearMonth = listDataYearMonthDay.get(i);
						File yearFolder = new File(folder,deleteDataYearMonth[0]);
						File monthFolder = new File(yearFolder,deleteDataYearMonth[1]);
						File dayFolder = new File(monthFolder,deleteDataYearMonth[2]);
						deleteFolder(dayFolder.getAbsolutePath(),result);
					}
					break;//日处理结束
				}
			}while(false);
			
			//清理空文件夹
			List<String> listDataYearFolder = listDataYearFolder(devicePath);
			for(String dataYear:listDataYearFolder) {
				List<String> listDataMonthFolder = listDataMonthFolder(devicePath, dataYear);
				for(String dataMonth:listDataMonthFolder) {
					List<String> listDataDayFolder = listDataDayFolder(devicePath, dataYear, dataMonth);
					//日都没了删除月
					if(listDataDayFolder.size() == 0) {
						File yearFolder = new File(folder,dataYear);
						File monthFolder = new File(yearFolder,dataMonth);
						deleteFolder(monthFolder.getAbsolutePath(),result);
					}
				}
				//月都没了删除年
				listDataMonthFolder = listDataMonthFolder(devicePath, dataYear);
				if(listDataMonthFolder.size() == 0) {
					File yearFolder = new File(folder,dataYear);
					deleteFolder(yearFolder.getAbsolutePath(),result);
				}
			}
			//年都没了删除设备
			listDataYearFolder = listDataYearFolder(devicePath);
			if(listDataYearFolder.size() == 0) {
				deleteFolder(folder.getAbsolutePath(),result);
			}
		} else {
			//未达到数据层
			if(folder.isDirectory()) {
				File[] listFiles = folder.listFiles();
				for(File file:listFiles) {
					cleanDataByDevicePath(file,devicePathIndexStart, cleanDataDevicePathSet, cleanDataTimeMode, keepDataNumber, result);
				}
			}
		}
	}
	
    public static void deleteFolder(String folderPath,CleanDataResult result) {
        try {
        	Files.walk(Paths.get(folderPath)).sorted((a, b) -> -a.toString().compareTo(b.toString())).forEach(p -> {
           	 try {
                    Files.delete(p);
                    result.setDletedNumber(result.getDletedNumber()+1);
                    LogUtil.info("删除路径成功：{}",p.toFile().getAbsolutePath());
                } catch (IOException e) {
					LogUtil.error("删除路径失败：{}",p.toFile().getAbsolutePath());
					LogUtil.error(e);
                }
           });
		} catch (Exception e) {
			LogUtil.error("删除路径失败：{}",folderPath);
			LogUtil.error(e);
		}
    }
	
	public StorageStatusPlus getTotalStorageStatus() throws Exception {
		File dataFileFolder = new File(dataFilePath);
		File dataFilePathFlag = new File(dataFileFolder,"dataFilePath.flag");
		if(!dataFilePathFlag.exists()) {
			return new StorageStatusPlus();
		}
		String content = FileUtil.simpleGetFileContent(dataFilePathFlag);
		return JsonUtil.fromJson(content, StorageStatusPlus.class);
	}
	
	public StorageStatusPlus getStorageStatus(String[] devicePathAry) throws Exception {
		StorageStatusPlus totalStatus = new StorageStatusPlus();
		for(String devicePath:devicePathAry) {
			File dataFileFolder = new File(dataFilePath,devicePath);
			File devicePathFlagFile = new File(dataFileFolder,"devicePath.flag");
			if(!devicePathFlagFile.exists()) {
				continue;
			}
			String content = FileUtil.simpleGetFileContent(devicePathFlagFile);
			StorageStatus status = JsonUtil.fromJson(content, StorageStatus.class);
			totalStatus.addTotalDataNumber(status.getTotalDataNumber());
			totalStatus.addTotalDataSize(status.getTotalDataSize());
		}
		
		return totalStatus;
	}
	
	public DataFileComponent() {}

	public int getIoSize() {
		return cache.listKeyLike("_FILE_RW:").size();
	}

	public DataFileComponent(String dataFilePath, long dataFileSizeMax) throws Exception {
		this.dataFilePath = dataFilePath;
		this.dataFileSizeMaxBytes = dataFileSizeMax;
		run();
		cache = new AxeCache("DFC-"+StringUtil.getRandomString(8));
	}

	public String getColSplitFlag() {
		return colSplitFlag;
	}

	public File getDataFile(String filePath) {
		File dataFile = new File(dataFilePath, filePath + ".data");
		if (!dataFile.exists()) {
			return null;
		}
		return dataFile;
	}
	
	public List<Map<String,Object>> listDataFilePath(String dataPath) {
		if(dataPath == null) {
			dataPath = "";
		}
		File[] listFiles = new File(dataFilePath,dataPath).listFiles();
		List<Map<String,Object>> result = new ArrayList<>();
		if(listFiles == null) {
			return result;
		}
		for(File file:listFiles) {
			Map<String,Object> map = new HashMap<>();
			map.put("fileName", file.getName());
			map.put("isDirectory", file.isDirectory());
			result.add(map);
		}
		return result;
	}
	
	public List<String> listDataYearFolder(String devicePath) {
		return listDeviceDataFolder(devicePath,"^[0-9]{4}$");
	}
	public List<String> listDataMonthFolder(String devicePath,String dataYear) {
		return listDeviceDataFolder(devicePath,"^[0-9]{2}$",dataYear);
	}
	public List<String> listDataDayFolder(String devicePath,String dataYear,String dataMonth) {
		return listDeviceDataFolder(devicePath,"^[0-9]{2}$",dataYear,dataMonth);
	}
	private List<String> listDeviceDataFolder(String devicePath, String dataFolderReg, String... dataFolderPath) {
		File deviceFolder = new File(dataFilePath, devicePath);
		List<String> result = new ArrayList<>();
		if (!deviceFolder.exists() || !deviceFolder.isDirectory()) {
			return result;
		}
		for (String dfp : dataFolderPath) {
			deviceFolder = new File(deviceFolder, dfp);
		}
		if (!deviceFolder.exists() || !deviceFolder.isDirectory()) {
			return result;
		}
		String[] nameList = deviceFolder.list();
		if (nameList.length > 0) {
			Pattern pattern = Pattern.compile(dataFolderReg);
			for (String name : nameList) {
				if (pattern.matcher(name).find()) {
					result.add(name);
				}
			}
		}
		//安从小到大顺序排
		result.sort(new Comparator<String>() {
			@Override
			public int compare(String o1, String o2) {
				return o1.compareTo(o2);
			}
		});
		return result;
	}

	public List<Integer> listIndex(String devicePath, String dataYear, String dataMonth, String dataDay) {
		File deviceDataDir = new File(dataFilePath, devicePath + "/" + dataYear + "/" + dataMonth + "/" + dataDay);
		Set<Integer> indexSet = new HashSet<>();
		if (deviceDataDir.exists() && deviceDataDir.isDirectory()) {
			String[] nameList = deviceDataDir.list();
			if (nameList.length > 0) {
				Pattern pattern = Pattern.compile("^[0-9]+\\.index$");
				for (String name : nameList) {
					if (pattern.matcher(name).find()) {
						indexSet.add(Integer.parseInt(name.substring(0, name.indexOf("."))));
					}
				}
			}
		}
		List<Integer> result = new ArrayList<>();
		result.addAll(indexSet);
		result.sort(new Comparator<Integer>() {
			@Override
			public int compare(Integer o1, Integer o2) {
				return o1.compareTo(o2);
			}
		});
		return result;
	}

	//多线程获取数据
	public List<Map<String, Object>> getDevicesDataListByTimestampMultiThread(String[] deviceColNamesArray,long startTimestamp, long durTimeMs, int count, String timestampFormat) throws Exception {
		//最多开1000数据个开一条线程
		int pageSize = 1000;
		int pages = count / pageSize;
		if (pages * pageSize < count) {
			pages++;
		}

		if (pages == 1) {
			//不足一页数据，不做多线程处理了
			return getDevicesDataListByTimestamp(deviceColNamesArray, startTimestamp, durTimeMs, count, timestampFormat);
		}

		CountDownLatch latch = new CountDownLatch(pages);
		ConcurrentHashMap<Integer, List<Map<String, Object>>> resultMap = new ConcurrentHashMap<>();
		for (int i = 0; i < pages; i++) {
			long threadStartTimestamp = startTimestamp + i * pageSize;
			int tmpCount = pageSize;
			if (i == pages - 1) {
				//最后一页
				tmpCount = count - i * pageSize;
			}

			int threadDataIndex = i;
			int threadCount = tmpCount;
			Thread thread = new Thread() {
				@Override
				public void run() {
					try {
						List<Map<String, Object>> dataList = getDevicesDataListByTimestamp(deviceColNamesArray,threadStartTimestamp, durTimeMs, threadCount, timestampFormat);
						resultMap.put(threadDataIndex, dataList);
					} catch (Exception e) {
						LogUtil.error(e);
					} finally {
						latch.countDown();
					}
				}
			};
			thread.start();
		}
		latch.await();

		List<Map<String, Object>> result = null;
		for (int i = 0; i < pages; i++) {
			if (i == 0) {
				result = resultMap.get(i);
			} else {
				result.addAll(resultMap.get(i));
			}
		}
		return result;
	}

	/**
	 * 单线程获取数据
	 * @param deviceColNamesArray	格式：["devicePath:colName1,colName2"]
	 * @param startTimestamp
	 * @param durTimeMs
	 * @param count
	 * @param timestampFormat
	 * @return
	 * @throws Exception
	 */
	public List<Map<String, Object>> getDevicesDataListByTimestamp(String[] deviceColNamesArray, long startTimestamp,long durTimeMs, int count, String timestampFormat) throws Exception {
		SimpleDateFormat dataDaySdf = new SimpleDateFormat("yyyy-MM-dd");
		SimpleDateFormat timestampSdf = null;
		if (StringUtil.isNotEmpty(timestampFormat)) {
			timestampSdf = new SimpleDateFormat(timestampFormat);
		}

		List<String[]> formatList = new ArrayList<>();
		for (String deviceColNames : deviceColNamesArray) {
			String[] split = deviceColNames.split(":");
			if (split.length != 2) {
				throw new LogableException("参数[deviceColNamesArray]的值格式错误：{}", deviceColNames);
			}
			for (String colName : split[1].split(",")) {
				String[] format = new String[] { split[0], colName };
				formatList.add(format);
			}
		}

		List<RandomAccessFile> ioList = new ArrayList<>();

		Map<String, String[]> dataRecordMap = new HashMap<>();
		Map<String, Long[]> minMaxTimestampMap = new HashMap<>();
		Map<String, Integer> dataRowCountMap = new HashMap<>();
		Map<String, Map<String, Integer>> colNameIndexMap = new HashMap<>();
		Map<String, RandomAccessFile> indexFileReaderMap = new HashMap<>();
		Map<String, RandomAccessFile> dataFileReaderMap = new HashMap<>();

		List<Map<String, Object>> result = new ArrayList<>();
		try {
			for (int i = 0; i < count; i++) {
				long timestamp = startTimestamp + i * durTimeMs;
				String[] dataYearMonthDay = dataDaySdf.format(new Date(timestamp)).split("-");
				String dataYear = dataYearMonthDay[0];
				String dataMonth = dataYearMonthDay[1];
				String dataDay = dataYearMonthDay[2];
				byte[] timestampBytes = ByteBuffer.allocate(8).putLong(timestamp).array();

				String[] colValues = new String[formatList.size()];
				for (int j = 0; j < formatList.size(); j++) {
					String[] format = formatList.get(j);
					String devicePath = format[0];
					String colName = format[1];

					int[] jAry = new int[] {j};
					colValues[j] = getDeviceDataData(ioList, dataRecordMap, minMaxTimestampMap, dataRowCountMap, colNameIndexMap, indexFileReaderMap, dataFileReaderMap, dataYear, dataMonth, dataDay, devicePath, colName, timestamp, timestampBytes, jAry);
					if(colValues[j] == null) {
						//说明没取到值
						try {
							if(colName.startsWith("eval(") && colName.endsWith(")")) {
								Map<String,String> params = new HashMap<>();
								Expression expression = JevalUtil.castExpression(colName.substring("eval".length()+1,colName.length()-1));
								for(Variable variable:expression.getVariables()) {
									String colNameTmp = variable.getRawName();
									String devicePathTmp = "";
									if(colNameTmp.indexOf(".") < 0) {//就是当前设备下的点位，直接省略了
										devicePathTmp = devicePath;
									}else {
										devicePathTmp = colNameTmp.substring(0,colNameTmp.indexOf("."));
										colNameTmp = colNameTmp.substring(colNameTmp.indexOf(".")+1);
									}
									
									String colValueTmp = getDeviceDataData(ioList, dataRecordMap, minMaxTimestampMap, dataRowCountMap, colNameIndexMap, indexFileReaderMap, dataFileReaderMap, dataYear, dataMonth, dataDay, devicePathTmp, colNameTmp, timestamp, timestampBytes, null);
									if(colValueTmp == null) {
										throw new LogableException("计算表达式计算失败：{}，{} 数据不存在",expression.getExpression(),devicePathTmp+"."+colNameTmp);
									}
									params.put(variable.getFlag(), colValueTmp);
								}
								colValues[j] = JevalUtil.calcu(expression.getExpression(), params,10);//小数点最多10位
							}
						} catch (Exception e) {
//							LogUtil.error(e.getMessage());
						}
					}
					
					//用新值覆盖
					j = jAry[0];
				}

				Map<String, Object> row = new HashMap<>();
				if (timestampSdf == null) {
					row.put("timestamp", timestamp);
				} else {
					row.put("timestamp", timestampSdf.format(new Date(timestamp)));
				}
				row.put("colValues", colValues);
				result.add(row);
			}
		} catch (Exception e) {
			LogUtil.error(e);
			throw e;
		} finally {
			for (RandomAccessFile io : ioList) {
				try {
					io.close();
				} catch (Exception e2) {
				}
			}
		}
		return result;
	}
	
	
	private String getDeviceDataData(
			List<RandomAccessFile> ioList,
			Map<String, String[]> dataRecordMap,
			Map<String, Long[]> minMaxTimestampMap,
			Map<String, Integer> dataRowCountMap,
			Map<String, Map<String, Integer>> colNameIndexMap,
			Map<String, RandomAccessFile> indexFileReaderMap,
			Map<String, RandomAccessFile> dataFileReaderMap,
			String dataYear,String dataMonth,String dataDay,
			String devicePath,String colName,
			long timestamp,
			byte[] timestampBytes,
			int[] j
			) throws Exception {
		if (!dataRecordMap.containsKey(devicePath + timestamp)) {
			//如果设备这个时间点没有查到过数据，则进行数据查询
			do {
				if (!minMaxTimestampMap.containsKey(devicePath)) {
					//跨天或者第一次，或者超过maxTimestamp了，都会取minMaxTimestamp
					List<Integer> indexList = listIndex(devicePath, dataYear, dataMonth, dataDay);
					for (int fileIndex : indexList) {
						File indexFile = new File(dataFilePath, devicePath + "/" + dataYear + "/" + dataMonth + "/" + dataDay + "/" + fileIndex + ".index");
						int dataRowCount = (int) (indexFile.length() / indexFileDataUnitLen);
						if (dataRowCount == 0) {
							//没数据
							continue;
						}
						RandomAccessFile tmpIndexFileReader = new RandomAccessFile(indexFile, "r");
						ioList.add(tmpIndexFileReader);
						int lowIndex = 0;
						int highIndex = dataRowCount - 1;
						byte[] lowTimestampBytes = getDataTimestampBytes(tmpIndexFileReader, lowIndex);
						ByteBuffer buffer = ByteBuffer.allocate(8).put(lowTimestampBytes);
						buffer.flip();
						long minTimestamp = buffer.getLong();
						byte[] highTimestampBytes = lowTimestampBytes;
						long maxTimestamp = minTimestamp;
						if (dataRowCount > 1) {
							highTimestampBytes = getDataTimestampBytes(tmpIndexFileReader, highIndex);
							buffer = ByteBuffer.allocate(8).put(highTimestampBytes);
							buffer.flip();
							maxTimestamp = buffer.getLong();
						}
						if (timestamp > maxTimestamp) {
							continue;
						}
						File dataFile = new File(dataFilePath, devicePath + "/" + dataYear + "/" + dataMonth + "/" + dataDay + "/" + fileIndex + ".data");
						if (!dataFile.exists()) {
							LogUtil.error("数据文件缺失，{}", dataFile.getAbsolutePath());
							continue;
						}

						//准备取数据要用的信息
						minMaxTimestampMap.put(devicePath, new Long[] { minTimestamp, maxTimestamp });
						dataRowCountMap.put(devicePath, dataRowCount);

						indexFileReaderMap.put(devicePath, tmpIndexFileReader);

						RandomAccessFile dataFileReader = new RandomAccessFile(dataFile, "r");
						ioList.add(dataFileReader);
						dataFileReaderMap.put(devicePath, dataFileReader);

						//读colName
						byte[] firstDataIndexBytes = new byte[4];
						tmpIndexFileReader.seek(0);
						tmpIndexFileReader.read(firstDataIndexBytes);
						buffer = ByteBuffer.allocate(4).put(firstDataIndexBytes);
						buffer.flip();
						int firstDataIndex = buffer.getInt();

						dataFileReader.seek(0);
						byte[] colNamesBytes = new byte[firstDataIndex - returnAndEnterBytes.length];
						dataFileReader.read(colNamesBytes);
						String[] findColNames = new String(colNamesBytes, "UTF-8").split(colSplitFlag);

						Map<String, Integer> colNameIndex = new HashMap<>();
						for (int index = 0; index < findColNames.length; index++) {
							colNameIndex.put(findColNames[index], index);
						}
						colNameIndexMap.put(devicePath, colNameIndex);
						break;
					}
				}

				if (!minMaxTimestampMap.containsKey(devicePath)) {
					//说明当日都没有数据文件块可以匹配
					minMaxTimestampMap.put(devicePath, null);
					break;
				}

				Long[] minMaxTimestamp = minMaxTimestampMap.get(devicePath);
				if (minMaxTimestamp == null) {
					//说明没有数据文件块可以匹配，不要浪费时间
					break;
				}

				//如果没到最小时间，则持续跳过
				if (timestamp < minMaxTimestamp[0]) {
					break;
				}
				//如果超过最大时间，则清空重来
				if (timestamp > minMaxTimestamp[1]) {
					minMaxTimestampMap.remove(devicePath);
					if(j != null && j.length == 1) {
						j[0]--;
					}
					break;
				}

				RandomAccessFile indexFileReader = indexFileReaderMap.get(devicePath);
				Integer dataRowIndex = null;
				if (timestamp == minMaxTimestamp[0]) {
					dataRowIndex = 0;
				} else if (timestamp == minMaxTimestamp[1]) {
					dataRowIndex = dataRowCountMap.get(devicePath) - 1;
				} else {
					//寻找具体的dataRowIndex
					//说明在low和high之间
					int low = 0;
					int high = dataRowCountMap.get(devicePath) - 1;
					while (true) {
						if (high - low <= 1) {
							break;
						}
						int middle = (high + low) / 2;
						int compare = compareTimestampBytes(timestampBytes,getDataTimestampBytes(indexFileReader, middle));
						if (compare < 0) {
							high = middle;
						} else if (compare == 0) {
							dataRowIndex = middle;
							break;
						} else {
							low = middle;
						}
					}
				}

				if (dataRowIndex == null) {
					//这个时间点，这台设备，找不到数据，所以给个空值，当前时间点的这个台设备的其他点位，不要尝试获取数据了
					dataRecordMap.put(devicePath + timestamp, null);
					break;
				}

				//万事俱备，拿数据
				RandomAccessFile dataFileReader = dataFileReaderMap.get(devicePath);

				DataRecordInfo dataRecordInfo = getDataRecordInfo(indexFileReader, dataRowIndex);

				//读colValue
				dataFileReader.seek(dataRecordInfo.getDataIndex());
				byte[] dataLine = new byte[dataRecordInfo.getDataLength()+returnAndEnterBytes.length];
				dataFileReader.read(dataLine);
				boolean dataLineIsOk = true;
				for(int k=0;k<returnAndEnterBytes.length;k++) {
					if(returnAndEnterBytes[k] != dataLine[dataRecordInfo.getDataLength()+k]) {
						dataLineIsOk = false;
						break;
					}
				}
				if(dataLineIsOk) {
					String[] findColValues = new String(dataLine,0,dataLine.length-returnAndEnterBytes.length, "UTF-8").split(colSplitFlag);
					//组装
					dataRecordMap.put(devicePath + timestamp, findColValues);
				}
			} while (false);
		}

		String[] dataRecord = dataRecordMap.get(devicePath + timestamp);
		if (dataRecord == null) {
			return null;
		}

		Integer colNameIndex = colNameIndexMap.get(devicePath).get(colName);
		if (colNameIndex == null) {
			return null;
		}
		return dataRecord[colNameIndex];
	}

	//多线程获取数据
	public DeviceDataListByTimestamp2 getDevicesDataListByTimestamp2MultiThread(String[] deviceColNamesArray,long startTimestamp, long durTimeMs, int count, String timestampFormat) throws Exception {
		//最多开1000数据个开一条线程
		int pageSize = 1000;
		int pages = count / pageSize;
		if (pages * pageSize < count) {
			pages++;
		}

		if (pages == 1) {
			//不足一页数据，不做多线程处理了
			return getDevicesDataListByTimestamp2(deviceColNamesArray, startTimestamp, durTimeMs, count, timestampFormat);
		}

		CountDownLatch latch = new CountDownLatch(pages);
		ConcurrentHashMap<Integer, DeviceDataListByTimestamp2> resultMap = new ConcurrentHashMap<>();
		for (int i = 0; i < pages; i++) {
			long threadStartTimestamp = startTimestamp + i * pageSize;
			int tmpCount = pageSize;
			if (i == pages - 1) {
				//最后一页
				tmpCount = count - i * pageSize;
			}

			int threadDataIndex = i;
			int threadCount = tmpCount;
			Thread thread = new Thread() {
				@Override
				public void run() {
					try {
						DeviceDataListByTimestamp2 dataListMap = getDevicesDataListByTimestamp2(deviceColNamesArray,threadStartTimestamp, durTimeMs, threadCount, timestampFormat);
						resultMap.put(threadDataIndex, dataListMap);
					} catch (Exception e) {
						LogUtil.error(e);
					} finally {
						latch.countDown();
					}
				}
			};
			thread.start();
		}
		latch.await();

		DeviceDataListByTimestamp2 result = null;
		for (int i = 0; i < pages; i++) {
			if (i == 0) {
				result = resultMap.get(i);
			} else {
				/**
				 * {deviceColNames:[1,2,3,4]}
				 */
				DeviceDataListByTimestamp2 pageData = resultMap.get(i);
				for (String key : pageData.getDeviceDataMap().keySet()) {
					result.getDeviceDataMap().get(key).addAll(pageData.getDeviceDataMap().get(key));
				}
				result.getTimesList().addAll(pageData.getTimesList());
				if(result.getFirstDataTimestamp() == null) {
					result.setFirstDataTimestamp(pageData.getFirstDataTimestamp());
				}else {
					if(pageData.getFirstDataTimestamp() != null) {
						if(result.getFirstDataTimestamp() > pageData.getFirstDataTimestamp()) {
							//取最小的
							result.setFirstDataTimestamp(pageData.getFirstDataTimestamp());
						}
					}
				}
				
				if(result.getLastDataTimestamp() == null) {
					result.setLastDataTimestamp(pageData.getLastDataTimestamp());
				}else {
					if(pageData.getLastDataTimestamp() != null) {
						if(result.getLastDataTimestamp() < pageData.getLastDataTimestamp()) {
							//取最大的
							result.setLastDataTimestamp(pageData.getFirstDataTimestamp());
						}
					}
				}
				
				if(result.getDurTimeMs() == null) {
					result.setDurTimeMs(pageData.getDurTimeMs());
				}
				
				if(result.getTimestampFormat() == null) {
					result.setTimestampFormat(pageData.getTimestampFormat());
				}
				
			}
		}
		return result;
	}

	/**
	 * 单线程获取数据
	 * @param deviceColNamesArray	格式：["devicePath:colName1,colName2"]
	 * @param startTimestamp
	 * @param durTimeMs
	 * @param count
	 * @param timestampFormat
	 * @return
	 * @throws Exception
	 */
	public DeviceDataListByTimestamp2 getDevicesDataListByTimestamp2(String[] deviceColNamesArray, long startTimestamp,long durTimeMs, int count, String timestampFormat) throws Exception {
		SimpleDateFormat dataDaySdf = new SimpleDateFormat("yyyy-MM-dd");
		SimpleDateFormat timestampSdf = null;
		if (StringUtil.isNotEmpty(timestampFormat)) {
			timestampSdf = new SimpleDateFormat(timestampFormat);
		}

		DeviceDataListByTimestamp2 result = new DeviceDataListByTimestamp2();
		result.setTimesList(new ArrayList<>());
		Map<String,List<String>> deviceDataMap = new HashMap<>();
		result.setDeviceDataMap(deviceDataMap);
		Long firstDataTimestamp = null;
		Long lastDataTimestamp = null;
		List<String[]> formatList = new ArrayList<>();
		for (String deviceColNames : deviceColNamesArray) {
			String[] split = deviceColNames.split(":");
			if (split.length != 2) {
				throw new LogableException("参数[deviceColNamesArray]的值格式错误：{}", deviceColNames);
			}
			for (String colName : split[1].split(",")) {
				String[] format = new String[] { split[0], colName, split[0]+":"+colName };
				formatList.add(format);
				deviceDataMap.put(format[2], new ArrayList<>());
			}
		}

		List<RandomAccessFile> ioList = new ArrayList<>();

		Map<String, String[]> dataRecordMap = new HashMap<>();
		Map<String, Long[]> minMaxTimestampMap = new HashMap<>();
		Map<String, Integer> dataRowCountMap = new HashMap<>();
		Map<String, Map<String, Integer>> colNameIndexMap = new HashMap<>();
		Map<String, RandomAccessFile> indexFileReaderMap = new HashMap<>();
		Map<String, RandomAccessFile> dataFileReaderMap = new HashMap<>();

		try {
			for (int i = 0; i < count; i++) {
				long timestamp = startTimestamp + i * durTimeMs;
				String[] dataYearMonthDay = dataDaySdf.format(new Date(timestamp)).split("-");
				String dataYear = dataYearMonthDay[0];
				String dataMonth = dataYearMonthDay[1];
				String dataDay = dataYearMonthDay[2];
				byte[] timestampBytes = ByteBuffer.allocate(8).putLong(timestamp).array();

				for (int j = 0; j < formatList.size(); j++) {
					String[] format = formatList.get(j);
					String devicePath = format[0];
					String colName = format[1];

					int[] jAry = new int[] {j};
					String colValue = getDeviceDataData(ioList, dataRecordMap, minMaxTimestampMap, dataRowCountMap, colNameIndexMap, indexFileReaderMap, dataFileReaderMap, dataYear, dataMonth, dataDay, devicePath, colName, timestamp, timestampBytes, jAry);
					if(colValue == null) {
						//说明没取到值
						try {
							if(colName.startsWith("eval(") && colName.endsWith(")")) {
								Map<String,String> params = new HashMap<>();
								Expression expression = JevalUtil.castExpression(colName.substring("eval".length()+1,colName.length()-1));
								for(Variable variable:expression.getVariables()) {
									String colNameTmp = variable.getRawName();
									String devicePathTmp = "";
									if(colNameTmp.indexOf(".") < 0) {//就是当前设备下的点位，直接省略了
										devicePathTmp = devicePath;
									}else {
										devicePathTmp = colNameTmp.substring(0,colNameTmp.indexOf("."));
										colNameTmp = colNameTmp.substring(colNameTmp.indexOf(".")+1);
									}
									
									String colValueTmp = getDeviceDataData(ioList, dataRecordMap, minMaxTimestampMap, dataRowCountMap, colNameIndexMap, indexFileReaderMap, dataFileReaderMap, dataYear, dataMonth, dataDay, devicePathTmp, colNameTmp, timestamp, timestampBytes, null);
									if(colValueTmp == null) {
										throw new LogableException("计算表达式计算失败：{}，{} 数据不存在",expression.getExpression(),devicePathTmp+"."+colNameTmp);
									}
									params.put(variable.getFlag(), colValueTmp);
								}
								colValue = JevalUtil.calcu(expression.getExpression(), params,10);//小数点最多10位
							}
						} catch (Exception e) {
//							LogUtil.error(e.getMessage());
						}
					}
					deviceDataMap.get(format[2]).add(colValue);
					
					//用新值覆盖
					j = jAry[0];
				}

				if (timestampSdf == null) {
					result.getTimesList().add(String.valueOf(timestamp));
				} else {
					result.getTimesList().add(timestampSdf.format(new Date(timestamp)));
				}
				if(firstDataTimestamp == null) {
					firstDataTimestamp = timestamp;
				}
				lastDataTimestamp = timestamp;
			}
		} catch (Exception e) {
			LogUtil.error(e);
			throw e;
		} finally {
			for (RandomAccessFile io : ioList) {
				try {
					io.close();
				} catch (Exception e2) {
				}
			}
		}
		result.setFirstDataTimestamp(firstDataTimestamp);
		result.setLastDataTimestamp(lastDataTimestamp);
		result.setDurTimeMs(durTimeMs);
		result.setTimestampFormat(timestampFormat);
		
		return result;
	}

	/**
	 * @param timestamp
	 * @param deviceColNamesArray  ["devicePath:colName1,colName2"]
	 * @return
	 * @throws Exception
	 */
	public Map<String, Map<String, String>> getDevicesDataByTimestamp(long timestamp, String[] deviceColNamesArray)
			throws Exception {
		SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
		String[] dataYearMonthDay = sdf.format(new Date(timestamp)).split("-");
		String dataYear = dataYearMonthDay[0];
		String dataMonth = dataYearMonthDay[1];
		String dataDay = dataYearMonthDay[2];
		Map<String, Map<String, String>> resultMap = new HashMap<>();
		byte[] timestampBytes = ByteBuffer.allocate(8).putLong(timestamp).array();
		for (String deviceColNames : deviceColNamesArray) {
			String[] split = deviceColNames.split(":");
			if (split.length != 2) {
				throw new LogableException("参数[deviceColNamesArray]的值格式错误：{}", deviceColNames);
			}
			String devicePath = split[0];
			String colNames = split[1];
			Map<String, String> deviceDataMap = getDeviceDataMapByTimestamp(devicePath, dataYear, dataMonth, dataDay, timestampBytes,colNames);
			resultMap.put(devicePath, deviceDataMap);
		}
		return resultMap;
	}

	private Map<String, String> getDeviceDataMapByTimestamp(String devicePath, String dataYear, String dataMonth, String dataDay, byte[] timestampBytes, String colNames) throws Exception {
		Map<String, String> resultMap = new HashMap<>();
		for (String colName : colNames.split(colSplitFlag)) {
			resultMap.put(colName, null);
		}
		List<Integer> indexList = listIndex(devicePath, dataYear, dataMonth, dataDay);
		DataRecordInfo dataRecordInfo = null;
		Integer dataBlockIndex = null;
		for (int index : indexList) {
			dataRecordInfo = getDataRecordInfo(devicePath, dataYear, dataMonth, dataDay, index, timestampBytes);
			if (dataRecordInfo != null) {
				dataBlockIndex = index;
				break;
			}
		}
		if (dataRecordInfo != null) {
			DataRecordPackage dataRecord = getDataRecord(devicePath, dataYear, dataMonth, dataDay, dataBlockIndex,dataRecordInfo);
			if (dataRecord == null) {
				return resultMap;
			}
			for (int i = 0; i < dataRecord.getDataBlockInfo().getColNames().length; i++) {
				String findColName = dataRecord.getDataBlockInfo().getColNames()[i];
				if (resultMap.containsKey(findColName)) {
					String findColValue = dataRecord.getRecord().getColValues()[i];
					resultMap.put(findColName, findColValue);
				}
			}
		}

		return resultMap;
	}

	private byte[] getDataTimestampBytes(RandomAccessFile indexFileReader, int rowIndex) throws IOException {
		indexFileReader.seek(rowIndex * indexFileDataUnitLen + 8);
		//计算数据时间戳
		byte[] dataTimestampBytes = new byte[8];
		indexFileReader.read(dataTimestampBytes);
		return dataTimestampBytes;
	}

	private int compareTimestampBytes(byte[] timestampBytes1, byte[] timestampBytes2) throws IOException {
		for (int i = 0; i < 8; i++) {
			if (timestampBytes1[i] < timestampBytes2[i]) {
				if (timestampBytes1[i] < 0 && timestampBytes2[i] >= 0) {
					return 1;
				} else {
					return -1;
				}
			} else if (timestampBytes1[i] > timestampBytes2[i]) {
				if (timestampBytes1[i] >= 0 && timestampBytes2[i] < 0) {
					return -1;
				} else {
					return 1;
				}
			}
		}
		return 0;
	}

	public Page pageSearch(String devicePath, String dataYear, String dataMonth, String dataDay, int dataBlockIndex, int pageNum, int pageSize)
			throws Exception {
		File indexFile = new File(dataFilePath, devicePath + "/" + dataYear + "/" + dataMonth + "/" + dataDay + "/" + dataBlockIndex + ".index");
		if (!indexFile.exists()) {
			throw new LogableException("索引文件缺失，{}", indexFile.getAbsolutePath());
		}
		long indexFileLength = indexFile.length();
		File dataFile = new File(dataFilePath, devicePath + "/" + dataYear + "/" + dataMonth + "/" + dataDay + "/" + dataBlockIndex + ".data");
		if (!dataFile.exists()) {
			throw new LogableException("数据文件缺失，{}", dataFile.getAbsolutePath());
		}
		long dataFileLastModified = dataFile.lastModified();
		long dataFileLength = dataFile.length();

		//前两列是idls固定给的列
		String[] colNames = FileUtil.simpleGetFileFirstLine(dataFile).split(colSplitFlag);
		DataBlockInfoPlus dataBlockInfo = buildDataBlockInfo(devicePath, dataYear, dataMonth, dataDay, dataFileLastModified, null, dataBlockIndex, indexFileLength, dataFileLength, colNames);
		int pages = dataBlockInfo.getDataRowCount() / pageSize;
		if (pages * pageSize < dataBlockInfo.getDataRowCount()) {
			pages++;
		}
		Page result = new Page();
		result.setPages(pages);
		//保持跟axe-mysql一致

		result.setPageConfig(new PageConfig(pageNum, pageSize));
		result.setCount(dataBlockInfo.getDataRowCount());
		List<DataRecordPlus> records = new ArrayList<>();
		result.setRecords(records);
		result.setDataBlockInfo(dataBlockInfo);

		RandomAccessFile indexFileReader = null;
		RandomAccessFile dataFileReader = null;
		try {
			indexFileReader = new RandomAccessFile(indexFile, "r");
			dataFileReader = new RandomAccessFile(dataFile, "r");

			long[] dataBlockStartEndTime = getDataBlockStartEndTime(indexFileReader, indexFileLength);
			dataBlockInfo.setDataStartTime(dataBlockStartEndTime[0]);
			dataBlockInfo.setDataEndTime(dataBlockStartEndTime[1]);

			if (pageNum > 0 && pageNum <= pages) {
				int startDataRecordRowIndex = (pageNum - 1) * pageSize;
				for (int i = 0; i < pageSize; i++) {
					//计算索引所在字节，从0开始
					int dataRecordRowIndex = startDataRecordRowIndex + i;
					if (dataRecordRowIndex >= result.getCount()) {
						break;
					}
					DataRecordInfo dataRecordInfo = getDataRecordInfo(indexFileReader, dataRecordRowIndex);
					dataFileReader.seek(dataRecordInfo.getDataIndex());
					byte[] dataLine = new byte[dataRecordInfo.getDataLength()+returnAndEnterBytes.length];
					dataFileReader.read(dataLine);
					boolean dataLineIsOk = true;
					for(int k=0;k<returnAndEnterBytes.length;k++) {
						if(returnAndEnterBytes[k] != dataLine[dataRecordInfo.getDataLength()+k]) {
							dataLineIsOk = false;
							break;
						}
					}
					if(dataLineIsOk) {
						String[] colValues = new String(dataLine,0,dataLine.length-returnAndEnterBytes.length, "UTF-8").split(colSplitFlag);
						DataRecordPlus record = new DataRecordPlus();
						record.setColValues(colValues);
						record.setRowNumber(dataRecordRowIndex + 1);//行号从1开始
						record.setTimestamp(dataRecordInfo.getTimestamp());
						records.add(record);
					}
				}
			}
		} catch (Exception e) {
			throw e;
		} finally {
			if (indexFileReader != null) {
				try {
					indexFileReader.close();
				} catch (Exception e2) {
				}
			}
			if (dataFileReader != null) {
				try {
					dataFileReader.close();
				} catch (Exception e2) {
				}
			}
		}

		return result;
	}

	public DataRecordPackage getLastRecord(String devicePath) throws Exception {
		List<String> yearList = listDataYearFolder(devicePath);
		if (CollectionUtil.isEmpty(yearList)) {
			return null;
		}
		return getLastRecord(devicePath, yearList.get(yearList.size()-1));
	}

	public DataRecordPackage getLastRecord(String devicePath,String dataYear) throws Exception {
		List<String> monthList = listDataMonthFolder(devicePath, dataYear);
		if (CollectionUtil.isEmpty(monthList)) {
			return null;
		}
		return getLastRecord(devicePath, dataYear, monthList.get(monthList.size()-1));
	}

	public DataRecordPackage getLastRecord(String devicePath,String dataYear,String dataMonth) throws Exception {
		List<String> dayList = listDataDayFolder(devicePath, dataYear, dataMonth);
		if (CollectionUtil.isEmpty(dayList)) {
			return null;
		}
		String dataDay = dayList.get(dayList.size()-1);
		return getLastRecord(devicePath, dataYear, dataMonth, dataDay);
	}
	
	public DataRecordPackage getLastRecord(String devicePath, String dataYear, String dataMonth, String dataDay) throws Exception {
		List<Integer> dataBlockIndexList = listIndex(devicePath, dataYear, dataMonth, dataDay);
		if (CollectionUtil.isEmpty(dataBlockIndexList)) {
			return null;
		}
		return getLastRecordFromDataBlock(devicePath, dataYear, dataMonth, dataDay, dataBlockIndexList.get(dataBlockIndexList.size() - 1));
	}
	
	public DataRecordPackage getLastRecord(String devicePath, String dataYear, String dataMonth, String dataDay, Integer dataBlockIndex) throws Exception {
		if(StringUtil.isNotEmpty(dataYear)) {
			if(StringUtil.isNotEmpty(dataMonth)) {
				if(StringUtil.isNotEmpty(dataDay)) {
					if(dataBlockIndex != null) {
						return getLastRecordFromDataBlock(devicePath, dataYear, dataMonth, dataDay, dataBlockIndex);
					} else {
						return getLastRecord(devicePath, dataYear, dataMonth, dataDay);
					}
				} else {
					return getLastRecord(devicePath, dataYear, dataMonth);
				}
			} else {
				return getLastRecord(devicePath, dataYear);
			}
		} else {
			return getLastRecord(devicePath);
		}
	}

	private DataRecordPackage getLastRecordFromDataBlock(String devicePath, String dataYear, String dataMonth, String dataDay, int dataBlockIndex) throws Exception {
		File indexFile = new File(dataFilePath, devicePath + "/" + dataYear+"/"+dataMonth+"/"+dataDay + "/" + dataBlockIndex + ".index");
		if (!indexFile.exists()) {
			throw new LogableException("索引文件缺失，{}", indexFile.getAbsolutePath());
		}
		long indexFileLength = indexFile.length();
		File dataFile = new File(dataFilePath, devicePath + "/" + dataYear+"/"+dataMonth+"/"+dataDay + "/" + dataBlockIndex + ".data");
		if (!dataFile.exists()) {
			throw new LogableException("数据文件缺失，{}", dataFile.getAbsolutePath());
		}
		long dataFileLastModified = dataFile.lastModified();
		long dataFileLength = dataFile.length();

		//起始行，从0开始
		RandomAccessFile indexFileReader = null;
		RandomAccessFile dataFileReader = null;
		try {
			indexFileReader = new RandomAccessFile(indexFile, "r");
			dataFileReader = new RandomAccessFile(dataFile, "r");

			long[] dataBlockStartEndTime = getDataBlockStartEndTime(indexFileReader, indexFileLength);

			//前两列是idls固定给的列
			String colNames = FileUtil.simpleGetFileFirstLine(dataFile);
			DataBlockInfoPlus dataBlockInfo = buildDataBlockInfo(devicePath, dataYear, dataMonth, dataDay, dataFileLastModified, dataBlockStartEndTime, dataBlockIndex, indexFileLength, dataFileLength,
					colNames.split(colSplitFlag));

			//计算索引所在字节
			DataRecordInfo dataRecordInfo = getDataRecordInfo(indexFileReader, dataBlockInfo.getDataRowCount() - 1);
			dataFileReader.seek(dataRecordInfo.getDataIndex());
			byte[] dataLine = new byte[dataRecordInfo.getDataLength()+returnAndEnterBytes.length];
			dataFileReader.read(dataLine);
			boolean dataLineIsOk = true;
			for(int i=0;i<returnAndEnterBytes.length;i++) {
				if(returnAndEnterBytes[i] != dataLine[dataRecordInfo.getDataLength()+i]) {
					dataLineIsOk = false;
					break;
				}
			}
			int goBack = 1;
			while(!dataLineIsOk) {
				if(dataBlockInfo.getDataRowCount() - 1 - goBack < 0) {
					break;
				}
				
				dataRecordInfo = getDataRecordInfo(indexFileReader, dataBlockInfo.getDataRowCount() - 1 - goBack );
				dataFileReader.seek(dataRecordInfo.getDataIndex());
				dataFileReader.read(dataLine);
				dataLineIsOk = true;
				for(int i=0;i<returnAndEnterBytes.length;i++) {
					if(returnAndEnterBytes[i] != dataLine[dataRecordInfo.getDataLength()+i]) {
						dataLineIsOk = false;
						break;
					}
				}
				
				goBack++;
			}
			
			if(dataLineIsOk) {
				String colValues = new String(dataLine,0,dataLine.length-returnAndEnterBytes.length, "UTF-8");
				DataRecordPlus record = new DataRecordPlus();
				record.setColValues(colValues.split(colSplitFlag));
				record.setRowNumber(dataBlockInfo.getDataRowCount());
				record.setTimestamp(dataRecordInfo.getTimestamp());

				DataRecordPackage drp = new DataRecordPackage();
				drp.setDataBlockInfo(dataBlockInfo);
				drp.setRecord(record);
				return drp;
			}else {
				return null;
			}
		} catch (Exception e) {
			throw e;
		} finally {
			if (indexFileReader != null) {
				try {
					indexFileReader.close();
				} catch (Exception e2) {
				}
			}
			if (dataFileReader != null) {
				try {
					dataFileReader.close();
				} catch (Exception e2) {
				}
			}
		}
	}

	private DataRecordInfo getDataRecordInfo(String devicePath, String dataYear, String dataMonth, String dataDay,
			int dataBlockIndex, byte[] timestampBytes) {
		File indexFile = new File(dataFilePath, devicePath + "/" + dataYear + "/" + dataMonth + "/" + dataDay + "/" + dataBlockIndex + ".index");
		if (!indexFile.exists()) {
			return null;
		}
		int rowNumber = (int) (indexFile.length() / indexFileDataUnitLen);
		if (rowNumber == 0) {
			return null;
		}

		DataRecordInfo result = null;
		do {
			RandomAccessFile indexFileReader = null;
			try {
				indexFileReader = new RandomAccessFile(indexFile, "r");
				int low = 0;
				int high = rowNumber - 1;
				int compare = compareTimestampBytes(timestampBytes, getDataTimestampBytes(indexFileReader, low));
				if (compare < 0) {
					//不在此dataBlockIndex里
					break;
				} else if (compare == 0) {
					//找到数据了
					result = getDataRecordInfo(indexFileReader, low);
					break;
				} else {
					compare = compareTimestampBytes(timestampBytes, getDataTimestampBytes(indexFileReader, high));
					if (compare > 0) {
						//不在此dataBlockIndex里
						break;
					} else if (compare == 0) {
						//找到数据了
						result = getDataRecordInfo(indexFileReader, high);
						break;
					} else {
						//说明在low和high之间
						Integer findRowIndex = null;
						while (true) {
							if (high - low <= 1) {
								break;
							}
							int middle = (high + low) / 2;
							compare = compareTimestampBytes(timestampBytes,
									getDataTimestampBytes(indexFileReader, middle));
							if (compare < 0) {
								high = middle;
							} else if (compare == 0) {
								findRowIndex = middle;
								break;
							} else {
								low = middle;
							}
						}
						if (findRowIndex != null) {
							result = getDataRecordInfo(indexFileReader, findRowIndex);
							break;
						}
					}
				}
			} catch (Exception e) {
				LogUtil.error(e);
			} finally {
				if (indexFileReader != null) {
					try {
						indexFileReader.close();
					} catch (Exception e2) {
					}
				}
			}
		} while (false);

		return result;
	}

	/**
	 * @param indexFileReader
	 * @param dataRowIndex	从0开始
	 * @return
	 * @throws Exception
	 */
	private DataRecordInfo getDataRecordInfo(RandomAccessFile indexFileReader, long dataRowIndex) throws Exception {
		long dataRowIndexInIndexFile = dataRowIndex * indexFileDataUnitLen;
		byte[] dataIndexBytes = new byte[4];
		indexFileReader.seek(dataRowIndexInIndexFile);
		indexFileReader.read(dataIndexBytes);
		//计算长度所在的字节
		byte[] dataLengthBytes = new byte[4];
		indexFileReader.seek(dataRowIndexInIndexFile + 4);
		indexFileReader.read(dataLengthBytes);
		//计算数据时间戳
		byte[] dataTimestampBytes = new byte[8];
		indexFileReader.seek(dataRowIndexInIndexFile + 8);
		indexFileReader.read(dataTimestampBytes);

		//根据索引和长度信息，拿行数据
		ByteBuffer byteBuffer = ByteBuffer.allocate(4).put(dataIndexBytes);
		byteBuffer.flip();
		int dataIndex = byteBuffer.getInt();
		byteBuffer = ByteBuffer.allocate(4).put(dataLengthBytes);
		byteBuffer.flip();
		int dataLength = byteBuffer.getInt();
		byteBuffer = ByteBuffer.allocate(8).put(dataTimestampBytes);
		byteBuffer.flip();
		long dataTimestamp = byteBuffer.getLong();

		DataRecordInfo dri = new DataRecordInfo();
		dri.setDataIndex(dataIndex);
		dri.setDataLength(dataLength);
		dri.setTimestamp(dataTimestamp);
		return dri;
	}

	public DataRecordPackage getDataRecord(String devicePath, String dataYear, String dataMonth, String dataDay, int dataBlockIndex, DataRecordInfo dataRecordInfo) throws Exception {
		File dataFile = new File(dataFilePath, devicePath + "/" + dataYear + "/" + dataMonth + "/" + dataDay + "/" + dataBlockIndex + ".data");
		if (!dataFile.exists()) {
			return null;
		}
		DataRecord dataRecord = null;
		do {
			RandomAccessFile dataFileReader = null;
			try {
				dataFileReader = new RandomAccessFile(dataFile, "r");
				dataFileReader.seek(dataRecordInfo.getDataIndex());
				byte[] dataLine = new byte[dataRecordInfo.getDataLength()+returnAndEnterBytes.length];
				dataFileReader.read(dataLine);
				boolean dataLineIsOk = true;
				for(int i=0;i<returnAndEnterBytes.length;i++) {
					if(returnAndEnterBytes[i] != dataLine[dataRecordInfo.getDataLength()+i]) {
						dataLineIsOk = false;
						break;
					}
				}
				if(dataLineIsOk) {
					String[] colValues = new String(dataLine,0,dataLine.length-returnAndEnterBytes.length, "UTF-8").split(colSplitFlag);
					dataRecord = new DataRecord();
					dataRecord.setColValues(colValues);
				}
			} catch (Exception e) {
			} finally {
				if (dataFileReader != null) {
					try {
						dataFileReader.close();
					} catch (Exception e2) {
					}
				}
			}
		} while (false);

		if (dataRecord == null) {
			return null;
		}
		String[] colNames = FileUtil.simpleGetFileFirstLine(dataFile).split(colSplitFlag);
		if (colNames.length != dataRecord.getColValues().length) {
			throw new LogableException("数据文件已损坏[{}/{}/{}.data]", devicePath, dataYear + "/" + dataMonth + "/" + dataDay, dataBlockIndex);
		}
		DataRecordPackage result = new DataRecordPackage();
		result.setRecord(dataRecord);
		DataBlockInfo dbi = new DataBlockInfo();
		dbi.setColNames(colNames);
		result.setDataBlockInfo(dbi);
		return result;
	}

	private long[] getDataBlockStartEndTime(RandomAccessFile indexFileReader, long indexFileSize) throws IOException {
		int dataRowCount = (int) (indexFileSize / indexFileDataUnitLen);
		if (dataRowCount <= 0) {
			return new long[] { 0, 0 };
		}

		//先取最后一条时间
		long dataBlockIndex = (dataRowCount - 1) * indexFileDataUnitLen;
		byte[] dataTimestampBytes = new byte[8];
		indexFileReader.seek(dataBlockIndex + 8);
		indexFileReader.read(dataTimestampBytes);

		//根据索引和长度信息，拿行数据
		ByteBuffer byteBuffer = ByteBuffer.allocate(8).put(dataTimestampBytes);
		byteBuffer.flip();
		long dataTimestampEnd = byteBuffer.getLong();

		//默认开始结束时间一样
		long dataTimestampStart = dataTimestampEnd;
		if (dataRowCount > 1) {
			//如果超过1条，则获取第一条记录时间作为开始时间
			dataBlockIndex = 0;
			indexFileReader.seek(dataBlockIndex + 8);
			indexFileReader.read(dataTimestampBytes);

			//根据索引和长度信息，拿行数据
			byteBuffer = ByteBuffer.allocate(8).put(dataTimestampBytes);
			byteBuffer.flip();
			dataTimestampStart = byteBuffer.getLong();
		}

		return new long[] { dataTimestampStart, dataTimestampEnd };
	}

	public void addData(String devicePath, String colNames, String colValues, Long timestamp)
			throws Exception {
		String[] colNamesSplit = colNames.split(colSplitFlag);
		if (colNamesSplit.length != colValues.split(colSplitFlag).length) {
			throw new LogableException("列与值的个数不一致！\r\n列：{}\r\n值：{}", colNames, colValues);
		}

		if (timestamp == null) {
			timestamp = System.currentTimeMillis();
		}
		
		SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
		String[] dataYearMonthDay = sdf.format(new Date(timestamp)).split("-");
		String dataYear = dataYearMonthDay[0];
		String dataMonth = dataYearMonthDay[1];
		String dataDay = dataYearMonthDay[2];

		File devicePathFolder = new File(dataFilePath, devicePath);
		File devicePathFolderParent = devicePathFolder.getParentFile();
		while(devicePathFolderParent != null) {
			File devicePathFlagFile = new File(devicePathFolderParent, "devicePath.flag");
			if(devicePathFlagFile.exists()) {
				throw new Exception("设备数据存储路径冲突");
			}
			devicePathFolderParent = devicePathFolderParent.getParentFile();
		}
		File deviceDataDir = new File(devicePathFolder, dataYear + "/" + dataMonth + "/" + dataDay);
		if (!deviceDataDir.exists() || !deviceDataDir.isDirectory()) {
			deviceDataDir.mkdirs();
		}
		//标志文件，表示到这一层，是设备数据层了
		File devicePathFlagFile = new File(devicePathFolder, "devicePath.flag");
		if(!devicePathFlagFile.exists()) {
			FileUtil.newFile(devicePathFlagFile, "");
		}

		File[] listFiles = deviceDataDir.listFiles();
		//找到最大文件号
		int dataBlockIndex = 0;
		for (File file : listFiles) {
			String name = file.getName();
			name = name.substring(0, name.indexOf("."));
			try {
				int nameNumber = Integer.parseInt(name);
				if (nameNumber > dataBlockIndex) {
					dataBlockIndex = nameNumber;
				}
			} catch (Exception e) {
				throw new LogableException("这个文件不该出现：{}", file.getAbsolutePath());
			}
		}

		String colNamesCacheKey = "COL_NAMES:" + devicePath + ":" + dataYear + ":" + dataMonth + ":" + dataDay + ":" + dataBlockIndex;
		if (dataBlockIndex > 0) {
			String colNamesBefore = cache.getValueByKey(colNamesCacheKey);
			File dataFileBefore = new File(deviceDataDir, dataBlockIndex + ".data");
			if (StringUtil.isEmpty(colNamesBefore)) {
				colNamesBefore = FileUtil.simpleGetFileFirstLine(dataFileBefore);
			}

			if (colNames.compareTo(colNamesBefore) != 0) {
				//说明数据列格式变了（表字段有增、删、改的情况）
				dataBlockIndex++;//得新编数据文件了
			}

			if (dataFileSizeMaxBytes > 0 && dataFileBefore.exists() && dataFileBefore.length() > dataFileSizeMaxBytes) {
				//如果数据文件，大小已经超过1G，由于io并不是rws实时更新到磁盘，因此此处文件大小存在一定滞后，关系不达
				dataBlockIndex++;//得新编数据文件了
			}
		} else {
			//说明新目录，从1开始
			dataBlockIndex++;//得新编数据文件了
		}
		cache.setKeyValue(colNamesCacheKey, colNames, cacheKeepTime);


		String lastTimestampCacheKey = "LAST_TIMESTAMP:" + devicePath + ":" + dataYear + ":" + dataMonth + ":" + dataDay + ":" + dataBlockIndex;;
		Long lastTimestamp = cache.getValueByKey(lastTimestampCacheKey);
		if(lastTimestamp == null) {
			try {
				DataBlockInfoPlus dataBlockInfo = getDataBlockInfo(devicePath, dataYear, dataMonth, dataDay, dataBlockIndex);
				if(dataBlockInfo != null) {
					lastTimestamp = dataBlockInfo.getDataEndTime();
					cache.setKeyValue(lastTimestampCacheKey, lastTimestamp, cacheKeepTime);
				}
			} catch (Exception e) {}
		}
		if(lastTimestamp != null) {
			if (lastTimestamp.compareTo(timestamp) > 0) {
				throw new LogableException("数据保存失败，数据时间戳禁止倒流，上报:{}<最后一条:{}", timestamp,lastTimestamp);
			}
			if (lastTimestamp.compareTo(timestamp) == 0) {
				throw new LogableException("数据保存失败，数据时间戳禁止重复，上报:{}=最后一条:{}", timestamp,lastTimestamp);
			}
		}
		
		ioCacheLock.lock();
		if(released) {
			throw new Exception("DataFileComponent Object is released, add data is faield!");
		}
		try {
			String indexFileWriterCacheKey = "INDEX_FILE_RW:" + devicePath + ":" + dataYear + ":" + dataMonth + ":" + dataDay + ":" + dataBlockIndex;
			OutputStream indexFileRw = cache.getValueByKey(indexFileWriterCacheKey);
			if (indexFileRw == null) {
				//索引文件
				//格式：4字节（startIndex）+4字节（长度）+8字节（时间戳）
				File indexFile = new File(deviceDataDir, dataBlockIndex + ".index");
				if(indexFile.length()%indexFileDataUnitLen > 0) {
					//说明文件出现损坏，去除尾部损坏部分继续
					RandomAccessFile raf = null;
					try {
						raf = new RandomAccessFile(indexFile, "rw");
			            raf.setLength(indexFile.length()-(indexFile.length()%indexFileDataUnitLen)); // 修改文件长度
					} catch (Exception e) {} finally {
						try {
							raf.close();
						} catch (Exception e2) {}
					}
				}
				
				indexFileRw = new FileOutputStream(indexFile, true);
//				LogUtil.info("+++ indexFileWriter create: {}", indexFileWriterCacheKey);
			}
			cache.setKeyValue(indexFileWriterCacheKey, indexFileRw, cacheKeepTime, new AxeCacheTimeoutListener() {
				@Override
				public void onTimeout(String key, Object value) {
//					LogUtil.info("--- indexFileWriter close: {}", key);
					try {
						((OutputStream)value).close();
					} catch (Exception e) {}
				}
			});

			String dataFileWriterCacheKey = "DATA_FILE_RW:" + devicePath + ":" + dataYear + ":" + dataMonth + ":" + dataDay + ":" + dataBlockIndex;
			OutputStream dataFileWriter = cache.getValueByKey(dataFileWriterCacheKey);
			if (dataFileWriter == null) {
				File dataFile = new File(deviceDataDir, dataBlockIndex + ".data");
				if(!dataFile.exists()) {
					//如果是新文件，则需要把列写在首行
					dataFileWriter = new FileOutputStream(dataFile);
					byte[] colNamesBytes = colNames.getBytes("utf-8");
					dataFileWriter.write(colNamesBytes);
					dataFileWriter.write(returnAndEnterBytes);
				}else {
					dataFileWriter = new FileOutputStream(dataFile, true);
				}
//				LogUtil.info("+++ dataFileWriter create: {}", dataFileWriterCacheKey);
			}
			cache.setKeyValue(dataFileWriterCacheKey, dataFileWriter, cacheKeepTime, new AxeCacheTimeoutListener() {
				@Override
				public void onTimeout(String key, Object value) {
//					LogUtil.info("--- dataFileWriter close: {}", key);
					try {
						((OutputStream)value).close();
					} catch (Exception e) {}
				}
			});

			//# 先获取数据文件现在的长度
			//数据文件，写入的起始下标，其实就是数据文件的长度，因为数据文件被限定不会超过1G大小，因此长度可以用整形来，4字节
			String dataFileLengthCacheKey = "DATA_FILE_LENGTH:" + devicePath + ":" + dataYear + ":" + dataMonth + ":" + dataDay + ":" + dataBlockIndex;
			Integer dataStartIndex = cache.getValueByKey(dataFileLengthCacheKey);
			if(dataStartIndex == null) {
				dataStartIndex = (int) new File(deviceDataDir, dataBlockIndex + ".data").length();
			}
			
			//# 再写入数据
			int newDataStartIndex = dataStartIndex;
			byte[] colValuesBytes = colValues.getBytes("utf-8");
			dataFileWriter.write(colValuesBytes);
			newDataStartIndex = newDataStartIndex+colValuesBytes.length;
			dataFileWriter.write(returnAndEnterBytes);
			newDataStartIndex = newDataStartIndex+returnAndEnterBytes.length;
			
			cache.setKeyValue(dataFileLengthCacheKey, newDataStartIndex, cacheKeepTime);

			//# 最后写入索引，这样如果索引写入没成功，不太会破坏结构
			//写入4字节的起始索引
			byte[] bytes = ByteBuffer.allocate(4).putInt(dataStartIndex).array();
			indexFileRw.write(bytes);
			//写入4字节的数据长度
			bytes = ByteBuffer.allocate(4).putInt(colValuesBytes.length).array();
			indexFileRw.write(bytes);
			//写入8字节的数据时间戳
			bytes = ByteBuffer.allocate(8).putLong(timestamp).array();
			indexFileRw.write(bytes);
			
			cache.setKeyValue(lastTimestampCacheKey, timestamp, cacheKeepTime);
		} catch (IOException e) {
			throw e;
		} finally {
			ioCacheLock.unlock();
		}
	}

	public DataBlockInfoPlus getDataBlockInfo(String devicePath, String dataYear, String dataMonth, String dataDay, int dataBlockIndex) throws Exception {
		File indexFile = new File(dataFilePath, devicePath + "/" + dataYear + "/" + dataMonth + "/" + dataDay + "/" + dataBlockIndex + ".index");
		if (!indexFile.exists()) {
			throw new LogableException("索引文件缺失，{}", indexFile.getAbsolutePath());
		}
		long indexFileLength = indexFile.length();
		File dataFile = new File(dataFilePath, devicePath + "/" + dataYear + "/" + dataMonth + "/" + dataDay + "/" + dataBlockIndex + ".data");
		if (!dataFile.exists()) {
			throw new LogableException("数据文件缺失，{}", dataFile.getAbsolutePath());
		}
		long dataFileLastModified = dataFile.lastModified();
		long dataFileLength = dataFile.length();

		long[] dataBlockStartEndTime = new long[] { 0, 0 };
		RandomAccessFile indexFileReader = null;
		try {

			indexFileReader = new RandomAccessFile(indexFile, "r");
			dataBlockStartEndTime = getDataBlockStartEndTime(indexFileReader, indexFileLength);
		} catch (IOException e) {
			throw e;
		} finally {
			if (indexFileReader != null) {
				try {
					indexFileReader.close();
				} catch (Exception e2) {
				}
			}
		}

		String colNames = FileUtil.simpleGetFileFirstLine(dataFile);
		return buildDataBlockInfo(devicePath, dataYear, dataMonth, dataDay, dataFileLastModified, dataBlockStartEndTime, dataBlockIndex, indexFileLength, dataFileLength, colNames.split(colSplitFlag));
	}
	
	public List<DataBlockInfoPlus> listDataBlockInfo(String devicePath,String dataYear,String dataMonth,String dataDay){
		List<Integer> listIndex = listIndex(devicePath,dataYear,dataMonth,dataDay);
		List<DataBlockInfoPlus> result = new ArrayList<>();
		for(Integer dataBlockIndex:listIndex) {
			try {
				DataBlockInfoPlus dataBlockInfo = getDataBlockInfo(devicePath,dataYear,dataMonth, dataDay, dataBlockIndex);
				result.add(dataBlockInfo);
			} catch (Exception e) {}
		}
		return result; 
	}

	private DataBlockInfoPlus buildDataBlockInfo(String devicePath, String dataYear, String dataMonth, String dataDay, long dataBlockUpdateTime,
			long[] dataStartEndTime, int dataBlockIndex, long indexFileSize, long dataFileSize, String[] colNames) {
		DataBlockInfoPlus result = new DataBlockInfoPlus();
		result.setDevicePath(devicePath);
		result.setDataYear(dataYear);
		result.setDataMonth(dataMonth);
		result.setDataDay(dataDay);
		if (dataStartEndTime != null && dataStartEndTime.length >= 2) {
			result.setDataStartTime(dataStartEndTime[0]);
			result.setDataEndTime(dataStartEndTime[1]);
		}
		result.setDataBlocUpdateTime(dataBlockUpdateTime);
		result.setDataBlockIndex(dataBlockIndex);
		result.setIndexFileSize(indexFileSize);
		result.setDataFileSize(dataFileSize);
		result.setDataRowCount((int) (indexFileSize / indexFileDataUnitLen));
		result.setColNames(colNames);
		return result;
	}

	@Override
	public int index() {
		return 0;
	}

	@Override
	public void run() throws Exception {
		File dataDir = new File(dataFilePath);
		if (!dataDir.exists() || !dataDir.isDirectory()) {
			dataDir.mkdirs();
		}

		if (dataFileSizeMaxBytes > Integer.MAX_VALUE) {
			LogUtil.error("dataFileSizeMax参数超过上限：{}", Integer.MAX_VALUE);
			System.exit(0);
		}
	}
	
}
